from langchain.document_loaders import TextLoader
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import FAISS

from config.embedding_config import get_openai_embeddings_local

loader = TextLoader(r"D:\model_code\pythonkonwledge\embeddings_\sidamingzhu.txt", encoding='utf-8')
docs = loader.load()

text_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=50)
data = text_splitter.split_documents(docs)
print(len(data))
for doc in data:
    print(len(doc.page_content), doc.page_content)

embeddings = get_openai_embeddings_local()

faiss_documents = FAISS.from_documents(data, embeddings, normalize_L2=True)

faiss_path = r"D:\model_code\pythonkonwledge\embeddings_\bert_\local\model1"

faiss_documents.save_local(faiss_path)






